#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :douban.py
# @Time      :2023/10/10 
# @Author    :CL
# @email     :1037654919@qq.com

import datetime
import random
import time
from multiprocessing import Pool
import pandas
import requests
from bs4 import BeautifulSoup
import os
import pandas as pd
from utils import MongoDBUtil, mongo_manager, get_proxy

MongoDBUtil = MongoDBUtil()
from retrying import retry


class CustomException(BaseException):
    def __init__(self, message):
        self.message = message

USER_AGENT_LIST=[
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1",
    "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 (KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5",
    "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 (KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3",
    "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)",
    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3",
    "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 (KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24",
    "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 (KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24"
]

headers ={
    "User-Agent":random.choice(USER_AGENT_LIST)}
class Douban():
    def __init__(self):
        self.url = "https://movie.douban.com/j/chart/top_list"
        self.headers = {
            "Accept": "*/*",
            "Accept-Language": "zh-CN,zh;q=0.9",
            "Connection": "keep-alive",
            "Referer": "https://movie.douban.com/typerank?",
            "Sec-Fetch-Dest": "empty",
            "Sec-Fetch-Mode": "cors",
            "Sec-Fetch-Site": "same-origin",
            "User-Agent":random.choice(USER_AGENT_LIST),
            "X-Requested-With": "XMLHttpRequest",
            "sec-ch-ua": "\"Not.A/Brand\";v=\"8\", \"Chromium\";v=\"114\", \"Google Chrome\";v=\"114\"",
            "sec-ch-ua-mobile": "?0",
            "sec-ch-ua-platform": "\"Linux\""
        }
        self.cookies = {
            # "bid": "IoWM2l63OaQ",
            # "_pk_id.100001.4cf6": "bfbb85ec3b2f9171.1693212023.",
            # "__yadk_uid": "pYhiu3ZErcq9rDud6uK3t6JzWmc43z39",
            # "ll": "\"118173\"",
            # "_ga": "GA1.1.1675583776.1693212024",
            # "_ga_RXNMP372GL": "GS1.1.1694317804.3.0.1694317804.60.0.0",
            # "ap_v": "0,6.0",
            # "_pk_ref.100001.4cf6": "%5B%22%22%2C%22%22%2C1696923072%2C%22https%3A%2F%2Fwww.baidu.com%2Flink%3Furl%3DJYQ6eBeq_jFR2myslmXH4riITyKYsa78nC_qTWlMi5x5XP3lC721vnoZE37g-E4msq2DOpLOpAPPjxuB8Lz4519VDFCjTszzzH-vmTMmbzu%26wd%3D%26eqid%3D8f289cca00000572000000066524fdb4%22%5D",
            # "_pk_ses.100001.4cf6": "1",
            # "__utma": "223695111.25768331.1693212024.1693212024.1696923072.2",
            # "__utmc": "223695111",
            # "__utmz": "223695111.1696923072.2.2.utmcsr=baidu|utmccn=(organic)|utmcmd=organic",
            # "__utmb": "30149280.4.10.1696923072",
            # "_pbjs_userid_consent_data": "3524755945110770",
            # "trc_cookie_storage": "taboola%2520global%253Auser-id%3Dfa2dcbc4-a26f-4801-b5ab-c3f67598bfb9-tuctc1e836f",
            # "_vwo_uuid_v2": "DA51A6B993D05A1420990E7FB770C589F|caf518610cd157b502246880d7d9ce45",
            # "cto_bundle": "m0Pu_F92a0FHeGx2N2lFRVVNakNuQ3hCVk1heG1lR3Q0TzNVNENKVSUyQkhRTmNZS2VUalFweTdwU1d5TGZ1WWRkJTJCSzBLc3FLNTV0cHZ4Y3lIJTJGUXhOSFo1Y3pTOGFkQUM0eEVPZXFFUTNZTTFReDM4M1pSV0ZURlZVQWklMkJNZFNDQ1VneW9DNHRIWnllaHRpanU0MjVaa3ZLaFI5dyUzRCUzRA",
            # "cto_bidid": "BgvXB19US2N3VnVWdSUyRk9lNGo4cXkxUER6RVhrdkhIWTg5Z0htYVhERnZwUVdHT3FlbE1KbmQlMkJvSGIlMkY2bFVlYW96Y082MFV0cEJsRkpNOTUwY2FFajJJNEJQJTJGSUR1NUNZZ2dnWFlmSiUyRmliN0I2azglM0Q",
            # "__gads": "ID=a186e19481dbd354-2220b5a030e30074:T=1696923106:RT=1696925266:S=ALNI_MZj0TQWjVO7K6H4Pt6vmVqubqp7jA",
            # "__gpi": "UID=00000a176f85104d:T=1696923106:RT=1696925266:S=ALNI_MZxxF-rEMhKk8Io26Z_eZqPXaXq9A"
        }
        self.proxies = get_proxy()

        self.proxies2 = {'http': '127.0.0.1:15732',
           'https': '127.0.0.1:15732'}

    def get_nlc(self,url = "http://read.nlc.cn/allSearch/searchList?searchType=1&showType=1&pageNo=20&searchWord=&classification=A"):
        # url = "http://read.nlc.cn/allSearch/searchList?searchType=1&showType=1&pageNo=20&searchWord=&classification=A"

        response = requests.get(url, headers=self.headers, cookies=self.cookies, proxies=self.proxies)

        # print(response.text)
        print(response)
        if response.status_code == 200:
            return response.text
        else:
            time.sleep(5)
            raise f'请求状态错误：{response.status_code}'
    # 获取各分类排行榜的总数
    @retry(stop_max_attempt_number=3)
    def get_total(self, params):
        url = "https://movie.douban.com/j/chart/top_list_count"
        # params = {
        #     "type": "25",
        #     "interval_id": "100:90"
        # }
        response = requests.get(url, headers=self.headers, cookies=self.cookies, proxies=self.proxies, params=params)

        print(response.text)
        print(response)
        if response.status_code == 200:
            return response.json()['total']
        else:
            time.sleep(5)
            raise f'请求状态错误：{response.status_code}'

    # 获取分类榜单
    @retry(stop_max_attempt_number=3)
    def get_type_list(self, params):
        url = "https://movie.douban.com/j/chart/top_list"
        # params = {
        #     "type": "25",
        #     "interval_id": "100:90",
        #     "action": "",
        #     "start": "0",
        #     "limit": "20"
        # }
        response = requests.get(url, headers=self.headers, cookies=self.cookies, proxies=self.proxies, params=params)

        # print(response.text)
        print(response)
        if response.status_code == 200:
            return response.json()
        else:
            time.sleep(5)
            raise f'请求状态错误：{response.status_code}'

    # 2022年豆瓣电影榜单
    @retry(stop_max_attempt_number=3)
    def get_movie_annual2022(self):
        url = "https://movie.douban.com/ithil_j/activity/movie_annual2022"
        params = {
            "with_widgets": "1"
        }
        response = requests.get(url, headers=self.headers, cookies=self.cookies, proxies=self.proxies, params=params)

        # print(response.text)
        print(response)
        if response.status_code == 200:
            return response.json()
        else:
            time.sleep(5)
            raise f'请求状态错误：{response.status_code}'

    # 获取电影信息
    @retry(stop_max_attempt_number=5)
    def get_movie_detail(self, url='https://movie.douban.com/subject/1291561/'):

        # url = 'https://movie.douban.com/subject/1291561/'
        response = requests.get(url, headers=self.headers, cookies=self.cookies, proxies=self.proxies2)
        # print(response.text)
        print(url,response)
        if response.status_code == 200:
            return response.text
        else:
            time.sleep(10)
            raise CustomException(f'请求状态错误：{response.status_code}')

    # 获取电影影评列表
    @retry(stop_max_attempt_number=3)
    def get_movie_reviews(self, url='https://movie.douban.com/subject/1291561/reviews'):
        response = requests.get(url, headers=self.headers, cookies=self.cookies, proxies=self.proxies)
        # print(response.text)
        print(url,response)
        if response.status_code == 200:
            return response.json()
        else:
            time.sleep(10)
            raise CustomException(f'请求状态错误：{response.status_code}')

    # 获取电影短评
    @retry(stop_max_attempt_number=3)
    def get_movie_comments(self, url='https://movie.douban.com/subject/1291561/comments?status=P'):
        urlc = 'https://movie.douban.com/subject/1291561/comments?status=P'
        urlc2 = 'https://movie.douban.com/subject/1291561/comments?start=20&limit=20&status=P&sort=new_score'
        pass

    # 获取电影论坛
    @retry(stop_max_attempt_number=3)
    def get_movie_discussion(self, url='https://movie.douban.com/subject/1291561/discussion/'):
        urld = 'https://movie.douban.com/subject/1291561/discussion/'
        urld2 = 'https://movie.douban.com/subject/1291561/discussion/?start=20&sort_by=time'

        pass

    # 获取电影新片榜单
    def get_new_movies(self):
        url = 'https://movie.douban.com/chart'
        response = requests.get(url, headers=self.headers, cookies=self.cookies)
        # print(response.text)
        return response


def main():
    print('main')
    # # 豆瓣新片榜
    # save_new_movie()

    # 获取豆瓣 type_code
    # type_code = get_type_code()

    # # 下载分类排行榜
    # run_type_list()

    # # 2022年豆瓣电影榜单
    # save_movie_annual2022()

    # # 电影详细信息
    # run_movie_detail()



if __name__ == "__main__":
    print()
    douban = Douban()
    main()

